package toutiao

import (
	//"encoding/json"
	"fmt"
	"github.com/astaxie/beego/httplib"
	"log"
	"math/rand"
	"reflect"
	"strconv"
	"time"
)

type ImageDetailData struct {
	MediaName   string `json:"media_name"`
	Title       string `json:"title"`
	ImageDetail struct {
		Url string `json:"url"`
	} `json:image_detail`
}

func GetSearchData(name string) {
	refererStr := "https://www.toutiao.com/search/?keyword="
	url := "https://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E5%A3%81%E7%BA%B8&autoload=true&count=20&cur_tab=1"
	var (
		offset int    = 0
		format string = "json"
		curtab int    = 1
		page   int    = 1
	)

	for {
		log.Printf(">>>正在爬取第%d页数据================", page)
		_offset := strconv.Itoa(offset)
		_curtab := strconv.Itoa(curtab)
		url = "https://www.toutiao.com/search_content/?offset=" + _offset +
			"&format=" + format + "&keyword=" + name + "&autoload=true&count=20&cur_tab=" + _curtab
		offset += 20

		req := httplib.Get(url)
		//req.Debug(true)
		//log.Println(req.Response())

		req.Header("Host", "www.toutiao.com")
		req.Header("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0")
		req.Header("Referer", refererStr+name)
		req.Header("X-Requested-With", "XMLHttpRequest")
		req.Header("Accept", "application/json, text/javascript")

		var res map[string]interface{}
		req.ToJSON(&res)

		if res == nil {
			break
		}

		var data []interface{}
		data = res["data"].([]interface{})
		log.Println(data)

		fmt.Println("data:", reflect.TypeOf(data))

		for k, v := range data {
			log.Println(k)
			//log.Println(v)

			d, ok := v.(map[string]interface{})
			if ok {
				log.Println(d["title"])
				log.Println(d["media_name"])
				log.Println(d["datetime"])
				//fmt.Println("title", reflect.TypeOf(d["title"]))

				if d["media_name"] == nil || d["title"] == nil || d["datetime"] == nil {
					break
				}

				for _, val := range d["image_detail"].([]interface{}) {
					m, o := val.(map[string]interface{})
					if o {
						log.Println(m["url"])
					}
				}
			}

		}

		log.Println("随机数：", rand.Intn(6)+6)
		//time.Sleep(6 * time.Second)
		page++
		//break
	}

	//rsp := httplib.Get(url)
	//req.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})

	//str, err := req.String()

	//log.Println(str)

	//log.Println(req.Response())
}

func CrawlList(k string, id int64) {
	referer := "https://www.toutiao.com/search/?keyword="
	//url := "https://www.toutiao.com/search_content/?offset=0&format=json&keyword=%E5%A3%81%E7%BA%B8&autoload=true&count=20&cur_tab=1"
	var (
		offset int    = 0
		format string = "json"
		curtab int    = 1
		page   int    = 1
	)

	imagesUrls := make([]string, 0)

	for i := 0; i < 9; i++ {
		log.Printf(">>>正在爬取第%d页数据================>>>", page)

		_offset := strconv.Itoa(offset)
		_curtab := strconv.Itoa(curtab)
		url := "https://www.toutiao.com/search_content/?offset=" + _offset +
			"&format=" + format + "&keyword=" + k + "&autoload=true&count=20&cur_tab=" + _curtab
		offset += 20

		log.Printf(">>>抓取的url(%s)", string(url))
		time.Sleep(2 * time.Second)
		req := httplib.Get(url)

		req.Header("Host", "www.toutiao.com")
		req.Header("User-Agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:31.0) Gecko/20100101 Firefox/31.0")
		req.Header("Referer", referer+k)
		req.Header("X-Requested-With", "XMLHttpRequest")
		req.Header("Accept", "application/json, text/javascript")

		//go get github.com/widuu/gojson
		var results map[string]interface{}
		req.ToJSON(&results)

		data, ok := results["data"]

		if ok {
			for _, v := range data.([]interface{}) {
				d, ok := v.(map[string]interface{})
				if ok {
					log.Println(reflect.TypeOf(d["article_url"]))
					log.Printf("%s\n", d["article_url"])
					log.Printf("%s\n", d["title"])
					log.Printf("%s\n", d["media_name"])

					dd, ok := d["image_detail"].([]interface{})
					if ok {
						for _, val := range dd {
							u, ok := val.(map[string]interface{})
							if ok {
								//log.Println(u["url"])
								log.Printf(">>>image:(%s\n", u["url"])
								u := fmt.Sprintf("%s", u["url"])
								imagesUrls = append(imagesUrls, u)

							}
						}
					}

				}

			}
		} else {
			return
		}

		r := rand.Intn(3) + 5
		time.Sleep(time.Duration(r) * time.Second)
		page++
	}

	log.Println(len(imagesUrls))

	for _, url := range imagesUrls {
		AddUrls(url, id)
	}

}
